x86: map frame table sparsely

Avoid backing frame table holes with memory, when those holes are
large enough to cover an exact multiple of large pages. This is based
on the introduction of a bit map, where each bit represents one such
range, thus allowing mfn_valid() checks to easily filter out those
MFNs that now shouldn't be used to index the frame table.

This allows for saving a couple of 2M pages even on "normal" systems.

Signed-off-by: Jan Beulich <jbeulich@novell.com>

diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index eefd3a27178786e67b5985c283c1eebf03f7942f..d47250fa40327c2bbe5f6acb5e0d783b431e5c50 100644 (file)
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -141,6 +141,10 @@ struct domain *dom_xen, *dom_io;
 unsigned long max_page;
 unsigned long total_pages;
 
+unsigned long __read_mostly pdx_group_valid[BITS_TO_LONGS(
+    (FRAMETABLE_SIZE / sizeof(*frame_table) + PDX_GROUP_COUNT - 1)
+    / PDX_GROUP_COUNT)] = { [0] = 1 };
+
 #define PAGE_CACHE_ATTRS (_PAGE_PAT|_PAGE_PCD|_PAGE_PWT)
 
 int opt_allow_hugepage;
@@ -162,36 +166,58 @@ l2_pgentry_t *compat_idle_pg_table_l2 = NULL;
 #define l3_disallow_mask(d) L3_DISALLOW_MASK
 #endif
 
-void __init init_frametable(void)
+static void __init init_frametable_chunk(void *start, void *end)
 {
-    unsigned long nr_pages, page_step, i, mfn;
-
-#ifdef __x86_64__
-    BUILD_BUG_ON(FRAMETABLE_VIRT_START & ((1UL << L3_PAGETABLE_SHIFT) - 1));
-    BUILD_BUG_ON(XEN_VIRT_END > FRAMETABLE_VIRT_END);
-#else
-    BUILD_BUG_ON(FRAMETABLE_VIRT_START & ((1UL << L2_PAGETABLE_SHIFT) - 1));
-#endif
-
-    nr_pages  = PFN_UP(max_pdx * sizeof(*frame_table));
-    page_step = 1 << (cpu_has_page1gb ? L3_PAGETABLE_SHIFT - PAGE_SHIFT
-                                      : L2_PAGETABLE_SHIFT - PAGE_SHIFT);
+    unsigned long s = (unsigned long)start;
+    unsigned long e = (unsigned long)end;
+    unsigned long step, mfn;
 
-    for ( i = 0; i < nr_pages; i += page_step )
+    ASSERT(!(s & ((1 << L2_PAGETABLE_SHIFT) - 1)));
+    for ( ; s < e; s += step << PAGE_SHIFT )
     {
+        step = 1UL << (cpu_has_page1gb &&
+                       !(s & ((1UL << L3_PAGETABLE_SHIFT) - 1)) ?
+                       L3_PAGETABLE_SHIFT - PAGE_SHIFT :
+                       L2_PAGETABLE_SHIFT - PAGE_SHIFT);
         /*
          * The hardcoded 4 below is arbitrary - just pick whatever you think
          * is reasonable to waste as a trade-off for using a large page.
          */
-        while (nr_pages + 4 - i < page_step)
-            page_step >>= PAGETABLE_ORDER;
-        mfn = alloc_boot_pages(page_step, page_step);
-        map_pages_to_xen(
-            FRAMETABLE_VIRT_START + (i << PAGE_SHIFT),
-            mfn, page_step, PAGE_HYPERVISOR);
+        while ( step && s + (step << PAGE_SHIFT) > e + (4 << PAGE_SHIFT) )
+            step >>= PAGETABLE_ORDER;
+        do {
+            mfn = alloc_boot_pages(step, step);
+        } while ( !mfn && (step >>= PAGETABLE_ORDER) );
+        if ( !mfn )
+            panic("Not enough memory for frame table");
+        map_pages_to_xen(s, mfn, step, PAGE_HYPERVISOR);
     }
 
-    memset(frame_table, 0, nr_pages << PAGE_SHIFT);
+    memset(start, 0, end - start);
+    memset(end, -1, s - (unsigned long)end);
+}
+
+void __init init_frametable(void)
+{
+    unsigned int sidx, eidx, nidx;
+    unsigned int max_idx = (max_pdx + PDX_GROUP_COUNT - 1) / PDX_GROUP_COUNT;
+
+#ifdef __x86_64__
+    BUILD_BUG_ON(XEN_VIRT_END > FRAMETABLE_VIRT_END);
+#endif
+    BUILD_BUG_ON(FRAMETABLE_VIRT_START & ((1UL << L2_PAGETABLE_SHIFT) - 1));
+
+    for ( sidx = 0; ; sidx = nidx )
+    {
+        eidx = find_next_zero_bit(pdx_group_valid, max_idx, sidx);
+        nidx = find_next_bit(pdx_group_valid, max_idx, eidx);
+        if ( nidx >= max_idx )
+            break;
+        init_frametable_chunk(pdx_to_page(sidx * PDX_GROUP_COUNT),
+                              pdx_to_page(eidx * PDX_GROUP_COUNT));
+    }
+    init_frametable_chunk(pdx_to_page(sidx * PDX_GROUP_COUNT),
+                          pdx_to_page(max_pdx - 1) + 1);
 }
 
 void __init arch_init_memory(void)

diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 599c8d0d0bf8cc4e31b2089097cdc6a6c1c130af..8b56868ca265a715391eab1fff1f38e9958fc4e2 100644 (file)
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -312,6 +312,16 @@ static void __init setup_max_pdx(void)
 #endif
 }
 
+static void __init set_pdx_range(unsigned long smfn, unsigned long emfn)
+{
+    unsigned long idx, eidx;
+
+    idx = pfn_to_pdx(smfn) / PDX_GROUP_COUNT;
+    eidx = (pfn_to_pdx(emfn - 1) + PDX_GROUP_COUNT) / PDX_GROUP_COUNT;
+    for ( ; idx < eidx; ++idx )
+        __set_bit(idx, pdx_group_valid);
+}
+
 /* A temporary copy of the e820 map that we can mess with during bootstrap. */
 static struct e820map __initdata boot_e820;
 
@@ -657,6 +667,8 @@ void __init __start_xen(unsigned long mbi_p)
         if ( (boot_e820.map[i].type != E820_RAM) || (s >= e) )
             continue;
 
+        set_pdx_range(s >> PAGE_SHIFT, e >> PAGE_SHIFT);
+
         /* Map the chunk. No memory will need to be allocated to do this. */
         map_pages_to_xen(
             (unsigned long)maddr_to_bootstrap_virt(s),
@@ -853,6 +865,8 @@ void __init __start_xen(unsigned long mbi_p)
         }
 #endif
 
+        set_pdx_range(s >> PAGE_SHIFT, e >> PAGE_SHIFT);
+
         /* Need to create mappings above 16MB. */
         map_s = max_t(uint64_t, s, 16<<20);
         map_e = e;

diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c
index 82f6d620cfc0a7b13d003728fa528a1de6711128..98d85d4cb0389ecbf844957dac9515bd39ec6b67 100644 (file)
--- a/xen/arch/x86/x86_64/mm.c
+++ b/xen/arch/x86/x86_64/mm.c
@@ -64,6 +64,14 @@ l3_pgentry_t __attribute__ ((__section__ (".bss.page_aligned")))
 l2_pgentry_t __attribute__ ((__section__ (".bss.page_aligned")))
     l2_xenmap[L2_PAGETABLE_ENTRIES];
 
+int __mfn_valid(unsigned long mfn)
+{
+    return likely(mfn < max_page) &&
+           likely(!(mfn & pfn_hole_mask)) &&
+           likely(test_bit(pfn_to_pdx(mfn) / PDX_GROUP_COUNT,
+                           pdx_group_valid));
+}
+
 void *alloc_xen_pagetable(void)
 {
     extern int early_boot;

diff --git a/xen/include/asm-x86/config.h b/xen/include/asm-x86/config.h
index 05ca0f90a5f8e17487c0f5fb29be22a257775831..d465c3b1f9fe51726e2302d59fb814735e258078 100644 (file)
--- a/xen/include/asm-x86/config.h
+++ b/xen/include/asm-x86/config.h
@@ -314,7 +314,8 @@ extern unsigned int video_mode, video_flags;
 #define RDWR_MPT_VIRT_END      LINEAR_PT_VIRT_START
 #define RDWR_MPT_VIRT_START    (RDWR_MPT_VIRT_END - (MACHPHYS_MBYTES<<20))
 #define FRAMETABLE_VIRT_END    RDWR_MPT_VIRT_START
-#define FRAMETABLE_VIRT_START  (FRAMETABLE_VIRT_END - (FRAMETABLE_MBYTES<<20))
+#define FRAMETABLE_SIZE         (FRAMETABLE_MBYTES<<20)
+#define FRAMETABLE_VIRT_START  (FRAMETABLE_VIRT_END - FRAMETABLE_SIZE)
 #define RO_MPT_VIRT_END                FRAMETABLE_VIRT_START
 #define RO_MPT_VIRT_START      (RO_MPT_VIRT_END - (MACHPHYS_MBYTES<<20))
 

diff --git a/xen/include/asm-x86/mm.h b/xen/include/asm-x86/mm.h
index f8c1e0131330bae68acb0df5db178ffbc759df8e..c569573446c87b6219e409b597f6953a34b22e66 100644 (file)
--- a/xen/include/asm-x86/mm.h
+++ b/xen/include/asm-x86/mm.h
@@ -263,6 +263,10 @@ extern unsigned long max_page;
 extern unsigned long total_pages;
 void init_frametable(void);
 
+#define PDX_GROUP_COUNT ((1 << L2_PAGETABLE_SHIFT) / \
+                         (sizeof(*frame_table) & -sizeof(*frame_table)))
+extern unsigned long pdx_group_valid[];
+
 /* Convert between Xen-heap virtual addresses and page-info structures. */
 static inline struct page_info *__virt_to_page(const void *v)
 {

diff --git a/xen/include/asm-x86/x86_32/page.h b/xen/include/asm-x86/x86_32/page.h
index 3cc4f1625fb60d50f82c05b8c01c9495b493d4a8..bae4d8ea2dde72334bd9dcb2172eb179d769f086 100644 (file)
--- a/xen/include/asm-x86/x86_32/page.h
+++ b/xen/include/asm-x86/x86_32/page.h
@@ -41,7 +41,11 @@
 #include <xen/config.h>
 #include <asm/types.h>
 
-#define __mfn_valid(mfn)        ((mfn) < max_page)
+#define __mfn_valid(mfn)        ({                                            \
+    unsigned long __m_f_n = (mfn);                                            \
+    likely(__m_f_n < max_page) &&                                             \
+    likely(test_bit(pfn_to_pdx(__m_f_n) / PDX_GROUP_COUNT, pdx_group_valid)); \
+})
 
 #define max_pdx                 max_page
 #define pfn_to_pdx(pfn)         (pfn)

diff --git a/xen/include/asm-x86/x86_64/page.h b/xen/include/asm-x86/x86_64/page.h
index 8bdf9b7365c4c534ec335041c1f641489d855fd4..78a79c59e3001ec4f7ffcb82c1c8e63511055947 100644 (file)
--- a/xen/include/asm-x86/x86_64/page.h
+++ b/xen/include/asm-x86/x86_64/page.h
@@ -35,7 +35,7 @@
 /* Physical address where Xen was relocated to. */
 extern unsigned long xen_phys_start;
 
-extern unsigned long max_page, max_pdx;
+extern unsigned long max_pdx;
 extern unsigned long pfn_pdx_bottom_mask, ma_va_bottom_mask;
 extern unsigned int pfn_pdx_hole_shift;
 extern unsigned long pfn_hole_mask;
@@ -53,10 +53,7 @@ extern void pfn_pdx_hole_setup(unsigned long);
 #define pdx_to_virt(pdx) ((void *)(DIRECTMAP_VIRT_START + \
                                    ((unsigned long)(pdx) << PAGE_SHIFT)))
 
-static inline int __mfn_valid(unsigned long mfn)
-{
-    return mfn < max_page && !(mfn & pfn_hole_mask);
-}
+extern int __mfn_valid(unsigned long mfn);
 
 static inline unsigned long pfn_to_pdx(unsigned long pfn)
 {